# -*- coding:utf-8 -*-
import os,sys
import re
import traceback
import time
from itertools import islice
from collections import deque
from prettytable import PrettyTable
sys.path.append(os.path.join(os.path.abspath(os.path.dirname(__file__)), os.pardir, os.pardir))
import supeanut_config
sys.path.append(os.path.join(os.path.abspath(os.path.dirname(__file__)), os.pardir))
from CommonLib.mylog import mylog
from CommonLib.db.MongoDBTool import MongoDBTool
from ExrightsTool import ExrightsTool
from TecIndexRecog import TecIndexRecog
from StockTool import StockTool
from TrainSetProduce import TrainSetProduce

'''
作者：supeanut
创建时间：2016-xx-xx xx:xx:xx
功能描述：
	形态匹配的统计或训练方式
相关配置：
	supeanut_config.XXX
历史改动：
	2016-xx-xx: xxxxxx
'''
class TrainSummary:
	def __init__(self, mem_mod=False, filename='trainSet_1.csv'):
		self.trainData = None
		self.combineData = {}
		self.mem_mod = mem_mod
		self.filename = filename
		self.db = MongoDBTool()
		if mem_mod is True:
			self.getTrainData()

	# train data form:
	# [code, datetime, f1, f2, ... label]
	def getTrainData(self, ):
		if self.mem_mod is False:
			return 
		dataObj = TrainSetProduce()
		trainData = dataObj.loadTrainSet(self.filename)
		self.trainData = trainData

	# 将所有相同的features归为一组，并记录(code, datetime, label)
	# "f1,f2,f3" ---> [(code, datetime, label), (code, datetime, label), ...]
	def combine(self, ):
		print 'combine same feature ...'
		if self.mem_mod is False:
			file_hd = open(self.filename, 'r')
			for line in islice(file_hd, 1, None):
				line_split = line.split(',')
				feature_str = "-".join(line_split[2: -1])
				if not self.combineData.has_key(feature_str):
					self.combineData[feature_str] = []
				self.combineData[feature_str].append(line_split[-1])
			return 
		for oneData in self.trainData:
			features_str = "-".join(oneData[2: -1])
			if not self.combineData.has_key(features_str):
				self.combineData[features_str] = []
			self.combineData[features_str].append(oneData[-1])

	def combine_userDefine(self, ):
		print 'combine userdefine feature ...'
		if self.mem_mod is False:
			file_hd = open(self.filename, 'r')
			for line in islice(file_hd, 1, None):
				line_split = line.split(',')
				feature_str = "all"
				if not self.combineData.has_key(feature_str):
					self.combineData[feature_str] = []
				self.combineData[feature_str].append(line_split[-1])
			return 
		for oneData in self.trainData:
			features_str = "all"
			if not self.combineData.has_key(features_str):
				self.combineData[features_str] = []
			self.combineData[features_str].append(oneData[-1])
		

	# 对combine数据进行简单的统计
	def summary_comebineData(self, ):
		file_hd = open("summary_comebineData_%s"%self.filename, 'w')
		print '对combine数据进行简单的统计:'
		csv_head_list = ['特征','总数','均值', '胜率',
			'<-5%','-5%~-2%','-2%~0%','0%~2%','2%~5%','>5%']
		file_hd.write(",".join(csv_head_list)+'\n')
		#print_table = PrettyTable(csv_head_list)
		for features_str, labels in self.combineData.items():
			# 处理总数
			labels_len = len(labels)
			if labels_len == 0:
				continue
			# 转float
			labels_float = []
			for label in labels:
				labels_float.append(float(label))
			# 处理总和
			labels_sum = sum(labels_float)
			# 处理均值
			labels_avg = labels_sum / labels_len
			# 处理简单统计
			labels_winner_num = 0
			# labels_fenbu_num : '<-5%','-5%~-2%','-2%~0%','0%~2%','2%~5%','>5%'
			labels_fenbu_num = [0, 0, 0, 0, 0, 0]
			for label in labels_float:
				if label >= 0.0:
					labels_winner_num += 1
				if label <= -0.05:
					labels_fenbu_num[0] += 1
				elif label <= -0.02:
					labels_fenbu_num[1] += 1
				elif label <= 0.0:
					labels_fenbu_num[2] += 1
				elif label <= 0.02:
					labels_fenbu_num[3] += 1
				elif label <= 0.05:
					labels_fenbu_num[4] += 1
				else:
					labels_fenbu_num[5] += 1
			labels_winrate = 1.0 * labels_winner_num / labels_len
			# 加入打印表格行
			one_summary = [features_str, 
				str(labels_len), 
				'%.5f'%labels_avg, 
				'%.5f'%labels_winrate, 
				'%.5f'%(1.0*labels_fenbu_num[0] / labels_len),
				'%.5f'%(1.0*labels_fenbu_num[1] / labels_len),
				'%.5f'%(1.0*labels_fenbu_num[2] / labels_len),
				'%.5f'%(1.0*labels_fenbu_num[3] / labels_len),
				'%.5f'%(1.0*labels_fenbu_num[4] / labels_len),
				'%.5f'%(1.0*labels_fenbu_num[5] / labels_len)]
			#print_table.add_row(one_summary)
			file_hd.write(','.join(one_summary)+'\n')
		file_hd.close()
		#print print_table
		
	def predict(self, codes=[], datetime="", train_filename="", output_filename="", mem_mod=False):
		print 'begin to predict ...'
		predict_result = []
		# 节省内存
		self.combineData = {}
		# 获得全市场历史表现
		file_hd = open("summary_comebineData_%s"%train_filename, 'r')
		all_stock_perform = {}
		for line in islice(file_hd, 1, None):
			line_split = line.split(',')
			features_str = line_split[0].replace('-',',')
			ocurs = int(line_split[1])
			avg_profit = float(line_split[2])
			win_rate = float(line_split[3])
			fenbu = [float(k) for k in line_split[4:10]]
			all_stock_perform[features_str] = [ocurs, avg_profit, win_rate, fenbu]
		file_hd.close()
		# 获得个股当日特征
		file_hd = open("calShape.temp",'r')
		#000099,2015-08-14 00:00:00,2,2,2,1,2,2,2,2,2,1,1,2,1,1,2,1,2,2,1,1,1,1,2,2,2,2,2,2,2,2,2,2,2,1,2,1,1,1,1,1,1,1
		stock_features_today = {}
		for line in islice(file_hd, 0, None):
			line_datetime = line[7:26]
			if line_datetime <> datetime:
				continue
			code = line[:6]
			features = line[27:-1]
			stock_features_today[code] = features
		file_hd.close()
		# 遍历个股及其历史表现
		file_hd = open("trainSet_allstock.csv", 'r')
		#000099,2015-06-12 00:00:00,2,2,2,1,2,2,2,2,2,1,2,2,1,2,2,2,2,2,1,1,1,2,2,2,2,2,2,2,2,2,2,2,2,1,1,1,1,1,1,1,2,1,-0.0923375363725
		code_summary = {}
		pre_code = ""
		for line in islice(file_hd, 1, None):
			cur_code = line[:6]
			if pre_code <> cur_code:
				if pre_code <> "":
					# -------------------评分--------------------------------
					# code=pre_code, code_summary={'features':[0.02,-0.01,]}, all_stock_perform={'features':[ocurs, avg_profit, win_rate, fenbu=[10%,12%,]]}
					today_features = stock_features_today.get(pre_code)
					# 过滤停牌等股票
					if today_features <> None:
						#print pre_code, today_features
						code_his_perform = code_summary.get(today_features)
						all_his_perform = all_stock_perform.get(today_features)
						# 计算个股胜率，均收益，分布统计
						stock_his = None
						if code_his_perform is not None:
							code_his_ocurs = len(code_his_perform)
							code_his_avg = 1.0* sum(code_his_perform) / code_his_ocurs
							win_num = 0
							code_his_fenbu = [0,0,0,0,0,0]
							for profit in code_his_perform:
								if profit > 0.0:
									win_num += 1
								if profit <= -0.05:
									code_his_fenbu[0] += 1
								elif profit <= -0.02:
									code_his_fenbu[1] += 1
								elif profit <= 0.0:
									code_his_fenbu[2] += 1
								elif profit <= 0.02:
									code_his_fenbu[3] += 1
								elif profit <= 0.05:
									code_his_fenbu[4] += 1
								else:
									code_his_fenbu[5] += 1
							for code_his_fenbu_i in range(0,6):
								code_his_fenbu[code_his_fenbu_i] = 1.0 * code_his_fenbu[code_his_fenbu_i] / code_his_ocurs
							code_his_winrate = 1.0* win_num / code_his_ocurs
							stock_his = [code_his_ocurs, code_his_avg, code_his_winrate, code_his_fenbu]
						score = self.stock_score(stock_his, all_his_perform)
						if score > 0.0:
							#print pre_code, score
							#print pre_code, today_features
							#print 'code', stock_his
							#print 'market', all_his_perform
							predict_result.append([score, pre_code, today_features, stock_his, all_his_perform])
					# -------------------以上，评分-------------------------
				code_summary = {}
			pre_code = cur_code
			# 开始个股统计
			last_douHao = line.rfind(',')
			features = line[27:last_douHao]
			label = float(line[last_douHao+1:-1])
			if not code_summary.has_key(features):
				code_summary[features] = []
			code_summary[features].append(label)
		predict_result.sort(key = lambda k:k[0], reverse =True)
		# 写入mongo
		self.db.conn('Supeanut','Predict')
		for predict_item in predict_result:
			print predict_item[1],predict_item[0]
			print predict_item[3]
			print predict_item[4]
		self.db.coll.update_one({'date':datetime[:10]},{'$set':{'predict':predict_result[:10]}}, upsert=True)
		file_hd.close()
		
	# 根据个股当前特征下的 个股/市场的历史数据，得出个股当前评分
	# 0:无历史数据，无评价； -1：不合格；False：错误；>0：评分数值
	def stock_score(self, stock_his, market_his):
		if stock_his is None and market_his is None:
			return 0
		if stock_his is None and market_his is not None:
			# 仅市场评分
			if market_his[0] < 100 or market_his[1] < 0.01 or market_his[2] < 0.55 or market_his[3][0] >0.15:
				return -1
			else:
				return market_his[1] * market_his[2]
		if stock_his is not None and market_his is not None:
			# 个股/市场 综合评分
			if (stock_his[0] >= 1 and stock_his[1] <0) or stock_his[2] <0.55 or stock_his[3][0] >= 0.2:
				return -1
			if stock_his[0] >= 2 and stock_his[1] >= 0.01 and stock_his[2] >= 0.6 and stock_his[3][0] <= 0.2 and stock_his[3][1] <= 0.25 and market_his[1] > 0.0 and market_his[2] > 0.5:
				return (stock_his[1] * stock_his[2] * market_his[1] * market_his[2]) ** 0.5
			if stock_his[0] == 1 and stock_his[1] < -0.02 and stock_his[1] > -0.05 and market_his[0] > 200 and market_his[2] >=0.6 and market_his[1] >=0.01 and market_his[3][0] <=0.1 and market_his[3][1]<=0.15:
				return ((0.5*market_his[2])**0.5)*market_his[1]
			if stock_his[0] == 1 and stock_his[1] >= -0.02 and stock_his[1] < 0.0 and market_his[0] > 100 and market_his[2] >=0.6 and market_his[1] >=0.01 and market_his[3][0] <=0.1 and market_his[3][1]<=0.15:
				return ((0.5*market_his[2])**0.5)*market_his[1]
			if stock_his[0] == 1 and stock_his[1] >= 0.0 and market_his[0] > 50 and market_his[2] >=0.6 and market_his[1] >=0.01 and market_his[3][0] <=0.1 and market_his[3][1]<=0.15:
				return (0.6 * stock_his[1] * market_his[1] * market_his[2]) ** 0.5
		return False

	def day_predict(self, today='2017-06-06'):
		#每日预测
		StockToolObj = StockTool()
		flag, codes = StockToolObj.get_all_stock_code('all')
		obj_data = TrainSetProduce(
				already_has_preData_inDesk=False,
				mem_mod=False,
				ignore_newstock_yiZiBan=True,
				filename="trainSet_allstock.csv",
				codes=codes,
				label_name='closeProfit', 
				period=['2007-01-01 00:00:00','2022-01-01 00:00:00'], 
				shapeList = ['MACDShape', 'MAShape'], 
				shapeParam = {'MACDShape':{}, 'MAShape':{}}
		)
		obj_data.calShape()
		obj_data.calLabel(winN=3)
		obj_data.CombineSet()
		obj_data.saveTrainSet()
		obj = TrainSummary(filename='trainSet_allstock.csv', mem_mod=False)
		obj.combine()
		obj.summary_comebineData()
		obj.predict(datetime="%s 00:00:00"%today, train_filename='trainSet_allstock.csv', output_filename="report.csv", mem_mod=False)
		
		

if __name__ == '__main__':
	obj = TrainSummary(filename='trainSet_allstock.csv', mem_mod=False)
	#obj = TrainSummary(filename='trainSet_onestock.csv', mem_mod=False)
	obj.combine()
	#obj.combine_userDefine()
	obj.summary_comebineData()
	exit()
	
	obj = TrainSummary(filename='trainSet_allstock.csv', mem_mod=False)
	#today = time.strftime('%Y-%m-%d',time.localtime(time.time()))
	today = '2017-06-08'
	obj.day_predict(today)
